{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Trains and Evaluates the IndianPines network using a feed dictionary\n",
    "========================================"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import time\n",
    "import numpy as np\n",
    "from six.moves import xrange  # pylint: disable=redefined-builtin\n",
    "import tensorflow as tf\n",
    "import os\n",
    "import IndianPinesCNN \n",
    "import patch_size\n",
    "# import IndianPines_data_set as input_data\n",
    "import Spatial_dataset as input_data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Declare model parameters as external flags\n",
    "-------------------------------------"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "flags = tf.app.flags\n",
    "FLAGS = flags.FLAGS\n",
    "flags.DEFINE_float('learning_rate', 0.01, 'Initial learning rate.')\n",
    "flags.DEFINE_integer('max_steps', 4000, 'Number of steps to run trainer.')\n",
    "flags.DEFINE_integer('conv1', 500, 'Number of filters in convolutional layer 1.')\n",
    "flags.DEFINE_integer('conv2', 100, 'Number of filters in convolutional layer 2.')\n",
    "flags.DEFINE_integer('hidden1', 200, 'Number of units in hidden layer 1.')\n",
    "flags.DEFINE_integer('hidden2', 84, 'Number of units in hidden layer 2.')\n",
    "flags.DEFINE_integer('batch_size', 100, 'Batch size.  '\n",
    "                     'Must divide evenly into the dataset sizes.')\n",
    "# flags.DEFINE_string('train_dir', '1.mat', 'Directory to put the training data.')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "learning_rate = 0.01\n",
    "num_epochs = 20\n",
    "max_steps = 4000\n",
    "IMAGE_SIZE = patch_size.patch_size\n",
    "conv1 = 500\n",
    "conv2 = 100\n",
    "fc1 = 200,\n",
    "fc2 = 84\n",
    "batch_size = 100\n",
    "TRAIN_FILES = 8\n",
    "TEST_FILES = 4\n",
    "DATA_PATH = os.path.join(os.getcwd(),\"Data\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def placeholder_inputs(batch_size):\n",
    "    \"\"\"Generate placeholder variables to represent the input tensors.\n",
    "    These placeholders are used as inputs by the rest of the model building\n",
    "    code and will be fed from the downloaded data in the .run() loop, below.\n",
    "    Args:\n",
    "    batch_size: The batch size will be baked into both placeholders.\n",
    "    Returns:\n",
    "    images_placeholder: Images placeholder.\n",
    "    labels_placeholder: Labels placeholder.\n",
    "    \"\"\"\n",
    "    # Note that the shapes of the placeholders match the shapes of the full\n",
    "    # image and label tensors, except the first dimension is now batch_size\n",
    "    # rather than the full size of the train or test data sets.\n",
    "    images_placeholder = tf.placeholder(tf.float32, shape=(batch_size, IndianPinesCNN\n",
    "                                                           .IMAGE_PIXELS))\n",
    "    labels_placeholder = tf.placeholder(tf.int32, shape=(batch_size))\n",
    "    return images_placeholder, labels_placeholder\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def fill_feed_dict(data_set, images_pl, labels_pl):\n",
    "    \"\"\"Fills the feed_dict for training the given step.\n",
    "    A feed_dict takes the form of:\n",
    "    feed_dict = {\n",
    "      <placeholder>: <tensor of values to be passed for placeholder>,\n",
    "      ....\n",
    "    }\n",
    "    Args:\n",
    "    data_set: The set of images and labels, from input_data.read_data_sets()\n",
    "    images_pl: The images placeholder, from placeholder_inputs().\n",
    "    labels_pl: The labels placeholder, from placeholder_inputs().\n",
    "    Returns:\n",
    "    feed_dict: The feed dictionary mapping from placeholders to values.\n",
    "    \"\"\"\n",
    "    # Create the feed_dict for the placeholders filled with the next\n",
    "    # `batch size ` examples.\n",
    "    images_feed, labels_feed = data_set.next_batch(batch_size)\n",
    "    feed_dict = {\n",
    "      images_pl: images_feed,\n",
    "      labels_pl: labels_feed,\n",
    "    }\n",
    "    return feed_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def do_eval(sess,\n",
    "            eval_correct,\n",
    "            images_placeholder,\n",
    "            labels_placeholder,\n",
    "            data_set):\n",
    "    \"\"\"Runs one evaluation against the full epoch of data.\n",
    "    Args:\n",
    "    sess: The session in which the model has been trained.\n",
    "    eval_correct: The Tensor that returns the number of correct predictions.\n",
    "    images_placeholder: The images placeholder.\n",
    "    labels_placeholder: The labels placeholder.\n",
    "    data_set: The set of images and labels to evaluate, from\n",
    "      input_data.read_data_sets().\n",
    "    \"\"\"\n",
    "    # And run one epoch of eval.\n",
    "    true_count = 0  # Counts the number of correct predictions.\n",
    "    steps_per_epoch = data_set.num_examples // batch_size\n",
    "    num_examples = steps_per_epoch * batch_size\n",
    "    for step in xrange(steps_per_epoch):\n",
    "        feed_dict = fill_feed_dict(data_set,\n",
    "                                   images_placeholder,\n",
    "                                   labels_placeholder)\n",
    "        true_count += sess.run(eval_correct, feed_dict=feed_dict)\n",
    "    precision = true_count / num_examples\n",
    "    print('  Num examples: %d  Num correct: %d  Precision @ 1: %0.04f' %\n",
    "        (num_examples, true_count, precision))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def add_DataSet(first,second):\n",
    "    temp_image = np.concatenate((first.images,second.images),axis=0)\n",
    "    temp_labels = np.concatenate((first.labels,second.labels),axis=0)\n",
    "    temp_image = temp_image.reshape(temp_image.shape[0],IMAGE_SIZE,IMAGE_SIZE,220)\n",
    "    temp_image = np.transpose(temp_image,(0,3,1,2))\n",
    "    temp_labels = np.transpose(temp_labels)\n",
    "    return input_data.DataSet(temp_image,temp_labels)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def run_training():\n",
    "    \"\"\"Train MNIST for a number of steps.\"\"\"\n",
    "    # Get the sets of images and labels for training, validation, and\n",
    "    # test on IndianPines.\n",
    "    \n",
    "    \"\"\"Concatenating all the training and test mat files\"\"\"\n",
    "    for i in range(TRAIN_FILES):\n",
    "        data_sets = input_data.read_data_sets(os.path.join(DATA_PATH, 'Train_'+str(IMAGE_SIZE)+'_'+str(i+1)+'.mat'), 'train')\n",
    "        if(i==0):\n",
    "            Training_data = data_sets\n",
    "            continue\n",
    "        else:\n",
    "            Training_data = add_DataSet(Training_data,data_sets)\n",
    "            \n",
    "    for i in range(TEST_FILES):\n",
    "        data_sets = input_data.read_data_sets(os.path.join(DATA_PATH, 'Test_'+str(IMAGE_SIZE)+'_'+str(i+1)+'.mat'),'test')\n",
    "        if(i==0):\n",
    "            Test_data = data_sets\n",
    "            continue\n",
    "        else:\n",
    "            Test_data = add_DataSet(Test_data,data_sets)\n",
    "        \n",
    "    # Tell TensorFlow that the model will be built into the default Graph.\n",
    "    with tf.Graph().as_default():\n",
    "    # Generate placeholders for the images and labels.\n",
    "        images_placeholder, labels_placeholder = placeholder_inputs(FLAGS.batch_size)\n",
    "\n",
    "        # Build a Graph that computes predictions from the inference model.\n",
    "        logits = IndianPinesCNN.inference(images_placeholder,\n",
    "                                 FLAGS.conv1,\n",
    "                                 FLAGS.conv2,        \n",
    "                                 FLAGS.hidden1,\n",
    "                                 FLAGS.hidden2)\n",
    "\n",
    "        # Add to the Graph the Ops for loss calculation.\n",
    "        loss = IndianPinesCNN.loss(logits, labels_placeholder)\n",
    "\n",
    "        # Add to the Graph the Ops that calculate and apply gradients.\n",
    "        train_op = IndianPinesCNN.training(loss, FLAGS.learning_rate)\n",
    "\n",
    "        # Add the Op to compare the logits to the labels during evaluation.\n",
    "        eval_correct = IndianPinesCNN.evaluation(logits, labels_placeholder)\n",
    "\n",
    "        # Build the summary operation based on the TF collection of Summaries.\n",
    "    #    summary_op = tf.merge_all_summaries()\n",
    "\n",
    "        # Add the variable initializer Op.\n",
    "        init = tf.initialize_all_variables()\n",
    "\n",
    "        # Create a saver for writing training checkpoints.\n",
    "        saver = tf.train.Saver()\n",
    "\n",
    "        # Create a session for running Ops on the Graph.\n",
    "        sess = tf.Session()\n",
    "\n",
    "        # Instantiate a SummaryWriter to output summaries and the Graph.\n",
    "    #    summary_writer = tf.train.SummaryWriter(FLAGS.train_dir, sess.graph)\n",
    "\n",
    "        # And then after everything is built:\n",
    "\n",
    "        # Run the Op to initialize the variables.\n",
    "        sess.run(init)\n",
    "\n",
    "        # Start the training loop.\n",
    "        for step in xrange(FLAGS.max_steps):\n",
    "            start_time = time.time()\n",
    "\n",
    "            # Fill a feed dictionary with the actual set of images and labels\n",
    "            # for this particular training step.\n",
    "            feed_dict = fill_feed_dict(Training_data,\n",
    "                                     images_placeholder,\n",
    "                                     labels_placeholder)\n",
    "\n",
    "            # Run one step of the model.  The return values are the activations\n",
    "            # from the `train_op` (which is discarded) and the `loss` Op.  To\n",
    "            # inspect the values of your Ops or variables, you may include them\n",
    "            # in the list passed to sess.run() and the value tensors will be\n",
    "            # returned in the tuple from the call.\n",
    "            _, loss_value = sess.run([train_op, loss],\n",
    "                                   feed_dict=feed_dict)\n",
    "\n",
    "            duration = time.time() - start_time\n",
    "\n",
    "            # Write the summaries and print an overview fairly often.\n",
    "            if step % 50 == 0:\n",
    "            # Print status to stdout.\n",
    "                print('Step %d: loss = %.2f (%.3f sec)' % (step, loss_value, duration))\n",
    "            # Update the events file.\n",
    "    #             summary_str = sess.run(summary_op, feed_dict=feed_dict)\n",
    "    #             summary_writer.add_summary(summary_str, step)\n",
    "    #             summary_writer.flush()\n",
    "\n",
    "            # Save a checkpoint and evaluate the model periodically.\n",
    "            if (step + 1) % 1000 == 0 or (step + 1) == FLAGS.max_steps:\n",
    "                saver.save(sess, 'model-spatial-CNN-'+str(IMAGE_SIZE)+'X'+str(IMAGE_SIZE)+'.ckpt', global_step=step)\n",
    "\n",
    "            # Evaluate against the training set.\n",
    "                print('Training Data Eval:')\n",
    "                do_eval(sess,\n",
    "                        eval_correct,\n",
    "                        images_placeholder,\n",
    "                        labels_placeholder,\n",
    "                        Training_data)\n",
    "                print('Test Data Eval:')\n",
    "                do_eval(sess,\n",
    "                        eval_correct,\n",
    "                        images_placeholder,\n",
    "                        labels_placeholder,\n",
    "                        Test_data)\n",
    "            # Evaluate against the validation set.\n",
    "    #             print('Validation Data Eval:')\n",
    "    #             do_eval(sess,\n",
    "    #                     eval_correct,\n",
    "    #                     images_placeholder,\n",
    "    #                     labels_placeholder,\n",
    "    #                     data_sets.validation)\n",
    "    #             # Evaluate against the test set.\n",
    "    #             print('Test Data Eval:')\n",
    "    #             do_eval(sess,\n",
    "    #                     eval_correct,\n",
    "    #                     images_placeholder,\n",
    "    #                     labels_placeholder,\n",
    "    #                     data_sets.test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Tensor(\"conv_1/Conv2D:0\", shape=(100, 9, 9, 500), dtype=float32)\n",
      "Tensor(\"h_conv2/h_conv2:0\", shape=(100, 3, 3, 100), dtype=float32)\n",
      "Tensor(\"h_pool2:0\", shape=(100, 2, 2, 100), dtype=float32)\n",
      "Tensor(\"Reshape:0\", shape=(100, 400), dtype=float32)\n",
      "Step 0: loss = 2.79 (0.417 sec)\n",
      "Step 50: loss = 2.73 (0.096 sec)\n",
      "Step 100: loss = 2.57 (0.084 sec)\n",
      "Step 150: loss = 2.14 (0.085 sec)\n",
      "Step 200: loss = 1.84 (0.096 sec)\n",
      "Step 250: loss = 1.58 (0.084 sec)\n",
      "Step 300: loss = 1.28 (0.084 sec)\n",
      "Step 350: loss = 1.21 (0.085 sec)\n",
      "Step 400: loss = 0.84 (0.085 sec)\n",
      "Step 450: loss = 1.04 (0.084 sec)\n",
      "Step 500: loss = 0.75 (0.084 sec)\n",
      "Step 550: loss = 0.83 (0.085 sec)\n",
      "Step 600: loss = 0.73 (0.084 sec)\n",
      "Step 650: loss = 0.72 (0.085 sec)\n",
      "Step 700: loss = 0.72 (0.085 sec)\n",
      "Step 750: loss = 0.61 (0.084 sec)\n",
      "Step 800: loss = 0.56 (0.347 sec)\n",
      "Step 850: loss = 0.59 (0.084 sec)\n",
      "Step 900: loss = 0.51 (0.084 sec)\n",
      "Step 950: loss = 0.51 (0.084 sec)\n",
      "Training Data Eval:\n",
      "  Num examples: 3200  Num correct: 2558  Precision @ 1: 0.7994\n",
      "Test Data Eval:\n",
      "  Num examples: 1600  Num correct: 851  Precision @ 1: 0.5319\n",
      "Step 1000: loss = 0.50 (0.088 sec)\n",
      "Step 1050: loss = 0.57 (0.080 sec)\n",
      "Step 1100: loss = 0.52 (0.079 sec)\n",
      "Step 1150: loss = 0.45 (0.080 sec)\n",
      "Step 1200: loss = 0.43 (0.080 sec)\n",
      "Step 1250: loss = 0.47 (0.080 sec)\n",
      "Step 1300: loss = 0.46 (0.080 sec)\n",
      "Step 1350: loss = 0.32 (0.080 sec)\n",
      "Step 1400: loss = 0.34 (0.081 sec)\n",
      "Step 1450: loss = 0.45 (0.080 sec)\n",
      "Step 1500: loss = 0.35 (0.080 sec)\n",
      "Step 1550: loss = 0.37 (0.079 sec)\n",
      "Step 1600: loss = 0.30 (0.339 sec)\n",
      "Step 1650: loss = 0.46 (0.080 sec)\n",
      "Step 1700: loss = 0.29 (0.080 sec)\n",
      "Step 1750: loss = 0.29 (0.080 sec)\n",
      "Step 1800: loss = 0.28 (0.080 sec)\n",
      "Step 1850: loss = 0.23 (0.080 sec)\n",
      "Step 1900: loss = 0.34 (0.079 sec)\n",
      "Step 1950: loss = 0.26 (0.080 sec)\n",
      "Training Data Eval:\n",
      "  Num examples: 3200  Num correct: 2982  Precision @ 1: 0.9319\n",
      "Test Data Eval:\n",
      "  Num examples: 1600  Num correct: 1183  Precision @ 1: 0.7394\n",
      "Step 2000: loss = 0.24 (0.087 sec)\n",
      "Step 2050: loss = 0.20 (0.079 sec)\n",
      "Step 2100: loss = 0.32 (0.080 sec)\n",
      "Step 2150: loss = 0.26 (0.080 sec)\n",
      "Step 2200: loss = 0.32 (0.080 sec)\n",
      "Step 2250: loss = 0.14 (0.080 sec)\n",
      "Step 2300: loss = 0.16 (0.080 sec)\n",
      "Step 2350: loss = 0.16 (0.080 sec)\n",
      "Step 2400: loss = 0.24 (0.340 sec)\n",
      "Step 2450: loss = 0.16 (0.080 sec)\n",
      "Step 2500: loss = 0.15 (0.080 sec)\n",
      "Step 2550: loss = 0.19 (0.079 sec)\n",
      "Step 2600: loss = 0.30 (0.080 sec)\n",
      "Step 2650: loss = 0.13 (0.080 sec)\n",
      "Step 2700: loss = 0.24 (0.080 sec)\n",
      "Step 2750: loss = 0.13 (0.080 sec)\n",
      "Step 2800: loss = 0.17 (0.081 sec)\n",
      "Step 2850: loss = 0.14 (0.080 sec)\n",
      "Step 2900: loss = 0.14 (0.080 sec)\n",
      "Step 2950: loss = 0.09 (0.080 sec)\n",
      "Training Data Eval:\n",
      "  Num examples: 3200  Num correct: 3095  Precision @ 1: 0.9672\n",
      "Test Data Eval:\n",
      "  Num examples: 1600  Num correct: 1356  Precision @ 1: 0.8475\n",
      "Step 3000: loss = 0.08 (0.090 sec)\n",
      "Step 3050: loss = 0.17 (0.080 sec)\n",
      "Step 3100: loss = 0.13 (0.080 sec)\n",
      "Step 3150: loss = 0.15 (0.080 sec)\n",
      "Step 3200: loss = 0.13 (0.341 sec)\n",
      "Step 3250: loss = 0.10 (0.080 sec)\n",
      "Step 3300: loss = 0.05 (0.080 sec)\n",
      "Step 3350: loss = 0.12 (0.084 sec)\n",
      "Step 3400: loss = 0.10 (0.080 sec)\n",
      "Step 3450: loss = 0.09 (0.080 sec)\n",
      "Step 3500: loss = 0.11 (0.080 sec)\n",
      "Step 3550: loss = 0.06 (0.080 sec)\n",
      "Step 3600: loss = 0.08 (0.080 sec)\n",
      "Step 3650: loss = 0.13 (0.080 sec)\n",
      "Step 3700: loss = 0.09 (0.082 sec)\n",
      "Step 3750: loss = 0.04 (0.087 sec)\n",
      "Step 3800: loss = 0.12 (0.082 sec)\n",
      "Step 3850: loss = 0.08 (0.087 sec)\n",
      "Step 3900: loss = 0.11 (0.079 sec)\n",
      "Step 3950: loss = 0.08 (0.086 sec)\n",
      "Training Data Eval:\n",
      "  Num examples: 3200  Num correct: 3144  Precision @ 1: 0.9825\n",
      "Test Data Eval:\n",
      "  Num examples: 1600  Num correct: 1408  Precision @ 1: 0.8800\n"
     ]
    }
   ],
   "source": [
    "run_training()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}
